package com.dmcb.trade.business.crawlers.author;

import com.dmcb.common.business.services.BaseService;
import com.dmcb.common.web.conversion.JsonResult;
import com.dmcb.common.business.utils.WebClientUtil;
import com.dmcb.trade.controller.ArticleCrawlerController;
import org.apache.commons.lang3.StringUtils;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;

/**
 * 品玩网文章全部导入
 * Created by wangyong on 2017/6/8.
 */
@Service
public class PinWanCrawler  extends BaseService {
    @Autowired
    private WebClientUtil webClientUtil;
    @Autowired
    private ArticleCrawlerController articleCrawlerController;
    org.slf4j.Logger logger = LoggerFactory.getLogger(PinWanCrawler.class);


    //目前以限制每个类别导入20篇文章
    public JsonResult crawl() {
        String[] types = new String[]{"figure","attitude","news","duanzi","knowledge","product","company"};
        int ero = 0;
        int success = 0;
        String link;
        for(String type : types){
            int page = 1;
            boolean isEnd = false;
            while (!isEnd) {
                link = "http://www.pingwest.com/category/"+type+"/page/"+page+"/";
                Document doc = webClientUtil.getDoc(link);
                if(doc == null ){
                    return error("解析失败，链接："+link);
                }
                Elements elements = doc.select("div.news-list>div");

                for (Element element : elements) {
                    String src = element.select("h2.title>a").attr("href");
                    if (StringUtils.isBlank(link)) {
                        continue;
                    }
                    JsonResult jsonResult = articleCrawlerController.crawl(3903, src);
                    if (jsonResult.getStatus() != JsonResult.STATUS_SUCCESS) {
                        if (jsonResult.getStatus() == JsonResult.STATUS_ERROR) {
                            ero++;
                        }
                        if (ero >= 10) {
                            return error("连续10次导入失败，请核实原因后再导入");
                        }
                        continue;
                    }
                    ero = 0;
                    success++;
                }
                page++;
                if (page == 3){
                    break;
                }
            }
        }

        return success("成功添加" + success + "条文章");
    }

}
